"""
1.url下载器----1.爬取url集合   2.未爬取url的集合（set去重复，防止链接重复爬取）
2.模块----
3.接口----1.判断是否有待取得url,方法定义has_new_url()
          2.添加新的url到未爬取集合中，方法定义add_new_url(url),add_new_urls(urls)
          3.获取一个未爬取的url,方法定义get_new_url()
          4.获取未爬取url集合的大小，方法定义new_url_size()
          5.获取已经爬取url集合的大小， old_url_size()
"""
class UrlManager(object):
    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()

    #1.判断是否有待取得url
    def has_new_url(self):
        return self.new_url_size() != 0

    #2.获取未爬取url集合的大小
    def new_url_size(self):
        return len(self.new_urls)

    #3.获取已经爬取url集合的大小
    def old_url_size(self):
        return len(self.old_urls)

    #4.获取一个未爬取的url
    def get_new_url(self):
        get_url = self.new_urls.pop()
        self.old_urls.add(get_url)
        return get_url

    def add_new_url(self, url):
        if url is None:
            return
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)




    # 5.添加新的url到未爬取集合中
    def add_new_urls(self, urls):
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)




